# Replication Archive for
# Coppock, Alexander. Generalizing from Survey Experiments Conducted on Mechanical Turk: A Replication Approach
# Forthcoming in Political Science Research and Methods

rm(list = ls())

# Uncomment to set working directory to replication archive
# setwd("")

# Uncomment to install packages
# install.packages("tidyverse")
# install.packages("metap")

library(tidyverse)

# Helper functions to implement randomization inference test.
source("coppock_generalizability_ri_functions.R")

# Load all datasets
load("coppock_generalizability_studies.rdata")

# set seed for reproducibility
set.seed(343)

# Study 1: Concealed Carry ------------------------------------------------

concealed_carry <- 
  concealed_carry_stacked %>%
  group_by(sample) %>% 
  do(homogenous_fx_test(dv_gun_s, Z_gun, data = .)$summary_df)

# Study 2: Immigration ----------------------------------------------------

immigration_1 <- 
  immigration_stacked %>%
  filter(Z_brader_pos_neg != "control") %>%
  group_by(sample) %>%
  do(homogenous_fx_test(brader_num_imm_s, Z_brader_pos_neg, data = .)$summary_df)

immigration_2 <- 
  immigration_stacked %>%
  filter(Z_brader_pos_neg != "control") %>%
  group_by(sample) %>%
  do(homogenous_fx_test(brader_neg_impact_s, Z_brader_pos_neg, data = .)$summary_df)

immigration <- 
  bind_rows(immigration_1,
            immigration_2)

# Study 3: Death Penalty --------------------------------------------------

death_penalty <- 
  death_penalty_stacked %>%
  filter(Z_CP_3 != "control") %>%
  group_by(sample) %>%
  do(homogenous_fx_test(CP_dv_s, Z_CP_3, data = .)$summary_df)

# Study 4: Superordinate Identity -----------------------------------------

superordinate_id_1 <-
  superordinate_id_stacked %>%
  group_by(sample) %>%
  do(homogenous_fx_test(willing_s, Z_identity, data = .)$summary_df)

superordinate_id_2 <-
  superordinate_id_stacked %>%
  group_by(sample) %>%
  do(homogenous_fx_test(willing_s, Z_particularism, data = .)$summary_df)

superordinate_id <-
  bind_rows(superordinate_id_1, superordinate_id_2)


# Study 5: Patriot Act ----------------------------------------------------

patriot_act <- 
  patriot_act_stacked %>%
  group_by(sample) %>% 
  do(homogenous_fx_test(PA_support_s, T1_condition_name, data = .)$summary_df)

# Study 6: Elite Endorsements ---------------------------------------------

# Note: must control for (or condition on) pid because of dif. probs. of assignment.
# Am combining using fisher's method metap::sumlog

elite_endorsements_results_1 <- 
  elite_endorsements_stacked %>%
  filter(Z_imm_match != "control") %>%
  group_by(pid_3, sample) %>% 
  do(homogenous_fx_test(imm_dv_s, Z_imm_match, data = .)$summary_df) %>%
  group_by(sample, condition_names, dv) %>%
  summarise(max_pval = metap::sumlog(max_pval)$p)

elite_endorsements_results_2 <- 
  elite_endorsements_stacked %>%
  filter(Z_fore_match != "control") %>%
  group_by(pid_3, sample) %>% 
  do(homogenous_fx_test(fore_dv_s, Z_fore_match, data = .)$summary_df)%>%
  group_by(sample, condition_names, dv) %>%
  summarise(max_pval = metap::sumlog(max_pval)$p)

elite_endorsements <- 
  bind_rows(elite_endorsements_results_1,
            elite_endorsements_results_2)

# Study 7: Mental Illness -------------------------------------------------

mental_illness_1 <- 
  mental_illness_stacked %>%
  group_by(sample) %>%
  do(homogenous_fx_test(mcginty_magazines_s, Z_mcginty_news, data = .)$summary_df)

mental_illness_2 <- 
  mental_illness_stacked %>%
  group_by(sample) %>%
  do(homogenous_fx_test(mcginty_magazines_s, Z_mcginty_policy, data = .)$summary_df)

mental_illness_3 <- 
  mental_illness_stacked %>%
  group_by(sample) %>%
  do(homogenous_fx_test(mcginty_SMI_danger_s, Z_mcginty_news, data = .)$summary_df)

mental_illness_4 <- 
  mental_illness_stacked %>%
  group_by(sample) %>%
  do(homogenous_fx_test(mcginty_SMI_danger_s, Z_mcginty_policy, data = .)$summary_df)

mental_illness <- 
  bind_rows(mental_illness_1,
            mental_illness_2,
            mental_illness_3,
            mental_illness_4)

# Study 8: System Threat --------------------------------------------------

system_threat_1 <- 
  system_threat_stacked %>%
  group_by(sample) %>%
  do(homogenous_fx_test(craig_num_imm_s, Z_craig, data = .)$summary_df)

system_threat_2 <- 
  system_threat_stacked %>%
  group_by(sample) %>%
  do(homogenous_fx_test(craig_wol_s, Z_craig, data = .)$summary_df)

system_threat <- 
  bind_rows(system_threat_1,
            system_threat_2)

# Study 9: Expert Economists ----------------------------------------------

expert_economists_1 <- 
  expert_economists_stacked %>%
  group_by(sample) %>%
  do(homogenous_fx_test(agree_1_s, Z_expert_1, data = .)$summary_df)

expert_economists_2 <- 
  expert_economists_stacked %>%
  group_by(sample) %>%
  do(homogenous_fx_test(agree_2_s, Z_expert_2, data = .)$summary_df)

expert_economists_3 <- 
  expert_economists_stacked %>%
  group_by(sample) %>%
  do(homogenous_fx_test(agree_3_s, Z_expert_3, data = .)$summary_df)

expert_economists_4 <- 
  expert_economists_stacked %>%
  group_by(sample) %>%
  do(homogenous_fx_test(agree_4_s, Z_expert_4, data = .)$summary_df)

expert_economists_5 <- 
  expert_economists_stacked %>%
  group_by(sample) %>%
  do(homogenous_fx_test(agree_5_s, Z_expert_5, data = .)$summary_df)

expert_economists <-
  bind_rows(expert_economists_1,
            expert_economists_2,
            expert_economists_3,
            expert_economists_4,
            expert_economists_5)

# Studies 10 and 11: Free Trade -------------------------------------------

free_trade_1 <- 
  free_trade_stacked %>%
  group_by(sample) %>% 
  do(homogenous_fx_test(Y_Hiscox_s, Z_Hiscox_expert, data = .)$summary_df)

free_trade_2 <- 
  free_trade_stacked %>%
  group_by(sample) %>% 
  do(homogenous_fx_test(Y_Hiscox_s, Z_Hiscox_valence, data = .)$summary_df)

free_trade <- 
  bind_rows(free_trade_1, free_trade_2)

# Studies 12 and 13: Polarization -----------------------------------------

polarization_1 <- 
  polarization_stacked %>%
  filter(Z_Levendusky != "placebo") %>%
  group_by(sample) %>%
  do(homogenous_fx_test(L_ex_s, Z_Levendusky, data = .)$summary_df)

polarization_2 <- 
  polarization_stacked %>%
  filter(Z_Levendusky != "placebo") %>%
  group_by(sample) %>%
  do(homogenous_fx_test(L_dif_s, Z_Levendusky, data = .)$summary_df)

polarization <- 
  bind_rows(polarization_1,
            polarization_2)

# Studies 14 and 15: Frame Breadth ----------------------------------------

frame_breadth_1 <- 
  frame_breadth_stacked %>%
  group_by(sample) %>% 
  do(homogenous_fx_test(Y_crime_s, Z_crime_Y_crime, data = .)$summary_df)
frame_breadth_2 <- 
  frame_breadth_stacked %>%
  group_by(sample) %>% 
  do(homogenous_fx_test(Y_health_s, Z_health_Y_health, data = .)$summary_df)
frame_breadth_3 <- 
  frame_breadth_stacked %>%
  group_by(sample) %>% 
  do(homogenous_fx_test(Y_stimulus_s, Z_stimulus_Y_stimulus, data = .)$summary_df)
frame_breadth_4 <- 
  frame_breadth_stacked %>%
  group_by(sample) %>% 
  do(homogenous_fx_test(Y_terror_s, Z_terror_Y_terror, data = .)$summary_df)

frame_breadth <- 
  bind_rows(frame_breadth_1,
            frame_breadth_2,
            frame_breadth_3,
            frame_breadth_4)


# Results -----------------------------------------------------------------

results <-
  bind_rows(
    concealed_carry = concealed_carry,
    immigration = immigration,
    death_penalty = death_penalty,
    superordinate_id = superordinate_id,
    patriot_act = patriot_act,
    elite_endorsements = elite_endorsements,
    mental_illness = mental_illness,
    system_threat = system_threat,
    expert_economists = expert_economists,
    free_trade = free_trade,
    polarization = polarization,
    frame_breadth = frame_breadth,
    .id = "study"
  )

save(results, file = "ri_results.rdata")
load("ri_results.rdata")

sink("coppock_generalizability_table_2.txt")
results %>%
  mutate(max_pval_adjust = p.adjust(max_pval, "holm")) %>%
  group_by(sample) %>%
  summarize(n = n(),
            n_sig = sum(max_pval <= 0.05),
            n_sig_adjust = sum(max_pval_adjust <= 0.05))
sink()


